work_dir="/data/git/gfdmp"
. ${work_dir}/function/logging.sh
. ${work_dir}/function/test_var.sh
. ${work_dir}/function/test_file_or_dir.sh
. ${work_dir}/function/test_dir_partition.sh

function f_mysql_recover()
{
	backup_tool="innobackupex"
	repl="$(awk -F= '{print $2}' <<< "${1}")" #moshan:判断是否需要建立连接
	recover_dir="$(awk -F= '{print $2}' <<< "${2}")"
	backup_dir="$(awk -F= '{print $2}' <<< "${3}")"
	current_recover_file="$(awk -F= '{print $2}' <<< "${4}")"
	mysql_admin_user="$(awk -F= '{print $2}' <<< "${5}")"
	mysql_admin_passwd="$(awk -F= '{print $2}' <<< "${6}")"
	f_test_var
	current_full_backup_dir="${recover_dir}"
	if [ "${current_recover_file}x" != "x" ]
	then
		#moshan:如果用户指定了current_recover_file变量的值, 则进行该步的处理
		f_test_file_or_dir "${backup_dir}" "file" "${current_recover_file}"
		mysql_recover_file=${test_file_or_dir}
		#moshan:将恢复的文件的路径取目录的路径, 如 /tmp/file --> /tmp
		current_full_backup_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_recover_file}")"
	else
		#moshan:如果用户未指定了current_recover_file变量的值, 则进行该步的处理
		if [ ! -d "${current_full_backup_dir}" ]
		then
			#moshan:判断/检查用户是否指定恢复目录, 若未指定, 则使用备份目录中最新的一份备份的目录
			current_full_backup_dir="${backup_dir}/$(ls -rtF ${backup_dir} 2>/dev/null|grep "/"|grep -v tmp|tail -1|sed 's#.$##g')"
		fi
		#moshan:根据备份目录路径, 找到备份文件, 并作为恢复的备份
		mysql_recover_file="$(ls -lrt ${current_full_backup_dir}/$(cut -c 1-17 <<< "${mysql_full_backup_file}")* 2>/dev/null|grep "^-"|tail -1|awk '{print $NF}')"
	fi
	f_logging "INFO" "MySQL data recovery starting, and will allocate ${use_mem} memory..."|tee -a ${log_file}
	f_logging "INFO" "MySQL full dir is \"${current_full_backup_dir}\"..." "1" "0"|tee -a ${log_file}
	#moshan:根据恢复文件, 找到整个备份的组号, 各个增备是按照全备的时间作为组号, 以此进行区分增备是以哪份全备来做的增备
	increm_group="$(grep $(cut -c 1-17 <<< "${mysql_full_backup_file}") <<< "${mysql_recover_file}"|awk -F'_' '{print $NF}')"
	#moshan:将所有的增备放到该数组
	incremental_arry=($(ls -rt ${current_full_backup_dir}|grep ${increm_group} 2>/dev/null|grep -v $(cut -c 1-17 <<< "${mysql_full_backup_file}")))
	mysql_conf_suffix="e6f2221a8f8774921ade1f814fb1c456" #moshan:定义mysql临时配置文件的后缀字符串
	old_mysql_conf="$(ls -lrt ${current_full_backup_dir}/my.cnf* 2> /dev/null|grep -v ${mysql_conf_suffix}|grep "^-"|tail -1|awk '{print $NF}')"
	#moshan:如果找不到mysql_recover_file或者mysql配置文件, 则退出本次恢复
	if [ "${mysql_recover_file}x" == "x" ]
	then
		f_logging "ERROR" "RECOVERY >> ERROR:No backup file!" "2" "0" |tee -a ${log_file}
		return
	elif [ "${old_mysql_conf}x" == "x" ]
	then
		f_logging "ERROR" "MySQL configuration file does not exist from full backup dir !" "2" "0"|tee -a ${log_file}
		return
	else
		old_mysql_port="${mysql_port}"
		if [ "${old_mysql_port}x" != "x" ]
		then
			#moshan:定义一个临时配置文件
			find /tmp -type f -name "*${mysql_conf_suffix}*"|xargs -i rm -f {}
			mysql_conf_tmp="${old_mysql_conf}_${mysql_conf_suffix}"
			#moshan:根据需要恢复的端口, 生成配置文件
			#f_var_initial
			server_id="$(awk -F. '{print $3$4}' <<< "${localhost_ip}")${mysql_recover_port}"
			sed "s#${old_mysql_port}#${mysql_recover_port}#g;s#^server.id.*#server_id=${server_id}#g" ${old_mysql_conf} > ${mysql_conf_tmp}
			#moshan:保存原来的配置文件的路径, 用于生成恢复的配置文件的路径, 恢复的原则就是以原来的为基准, 仅将路径中的端口改掉
			mysql_conf_path="${mysql_conf}"
			#moshan:恢复的配置文件
			old_mysql_data_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "datadir"|tr -d " \t"|awk -F= '{print $2}')"
			old_mysql_sock="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "socket"|tail -1|tr -d " \t"|awk -F= '{print $2}')"
			old_mysql_pid="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "pid[-_]file"|tail -1|tr -d " \t"|awk -F= '{print $2}')"
			old_mysql_tmp_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "tmpdir"|tr -d " \t"|awk -F= '{print $2}')"
			old_mysql_undo_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "innodb[_-]undo[_-]directory"|tr -d " \t"|awk -F= '{print $2}')"
			
			mysql_conf="${mysql_conf_tmp}"
			mysql_sock="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "socket"|tail -1|tr -d " \t"|awk -F= '{print $2}')"
			mysql_pid="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "pid[-_]file"|tail -1|tr -d " \t"|awk -F= '{print $2}')"
			mysql_data_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "datadir"|tr -d " \t"|awk -F= '{print $2}')"
			mysql_tmp_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "tmpdir"|tr -d " \t"|awk -F= '{print $2}')"
			mysql_undo_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "innodb[_-]undo[_-]directory"|tr -d " \t"|awk -F= '{print $2}')"
			mysql_base_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "basedir"|tr -d " \t"|awk -F= '{print $2}')"
			mysql_binlog_dir="$(sed 's#^ ##g' ${mysql_conf} 2>/dev/null|grep -v "^#"|grep -i "log[_-]bin"|tr -d " \t"|awk -F= '{print $2}'|awk -F'/' 'OFS="/",NF-=1')"
			if [ "${old_mysql_port}x" != "${mysql_recover_port}x" ]
			then
				#moshan:如果需要恢复的端口实例, 与备份的配置文件的端口实例不是一个端口
				#moshan:但是他们的数据目录又是一样的, 则视为异常情况, 不继续恢复
				if [ "${old_mysql_data_dir}x" == "${mysql_data_dir}x" ]
				then
					f_logging "ERROR" "The port [${mysql_recover_port}] that mysql needs to recover is different from the port [${old_mysql_port}] of the backup configuration file, but their data directory is the same, and EXIT" "2" "0"|tee -a ${log_file}
					return
				fi
				if [ "${old_mysql_sock}x" == "${mysql_sock}x" -a "$(grep -c "^/" <<< "${mysql_sock}")x" == "1x" ]
				then
					#moshan:判断socket 文件是否一致, 仅比对绝对路径下
					f_logging "ERROR" "The port [${mysql_recover_port}] that mysql needs to recover is different from the port [${old_mysql_port}] of the backup configuration file, but their socket file is the same, and EXIT" "2" "0"|tee -a ${log_file}
					return
				fi
				if [ "${old_mysql_pid}x" == "${mysql_pid}x" -a "$(grep -c "^/" <<< "${mysql_pid}")x" == "1x" ]
				then
					#moshan:判断pid 文件是否一致, 仅比对绝对路径下
					f_logging "ERROR" "The port [${mysql_recover_port}] that mysql needs to recover is different from the port [${old_mysql_port}] of the backup configuration file, but their pid file is the same, and EXIT" "2" "0"|tee -a ${log_file}
					return
				fi
				if [ "${old_mysql_tmp_dir}x" == "${mysql_tmp_dir}x" -a "$(grep -c "^/" <<< "${mysql_tmp_dir}")x" == "1x" ]
				then
					#moshan:判断tmp目录是否一致, 仅比对绝对路径下
					f_logging "ERROR" "The port [${mysql_recover_port}] that mysql needs to recover is different from the port [${old_mysql_port}] of the backup configuration file, but their tmp dir is the same, and EXIT" "2" "0"|tee -a ${log_file}
					return
				fi
				if [ "${old_mysql_undo_dir}x" == "${mysql_undo_dir}x" -a "$(grep -c "^/" <<< "${mysql_undo_dir}")x" == "1x" ]
				then
					#moshan:判断undo目录是否一致, 仅比对绝对路径下
					f_logging "ERROR" "The port [${mysql_recover_port}] that mysql needs to recover is different from the port [${old_mysql_port}] of the backup configuration file, but their undo dir is the same, and EXIT" "2" "0"|tee -a ${log_file}
					return
				fi
			elif [ "${old_mysql_port}x" == "${mysql_recover_port}x" ]
			then
				if [ "${background_operation}x" == "1x" ]
				then
					f_logging "WARN" "The instance port[${mysql_recover_port}] to be restored is the same as the original instance port[${old_mysql_port}]. But background_operation=1, and the recovery operation continues. " "2" "0"
				else
					f_logging "ERROR" "The instance port[${mysql_recover_port}] to be restored is the same as the original instance port[${old_mysql_port}]. But background_operation=0, and terminate this recovery operation" "2" "1"
					exit
				fi
			else
				f_logging "WARN" "The instance port[${mysql_recover_port}] to be restored is the same as the original instance port[${old_mysql_port}]. Please enter your choice." "2" "0"
				f_logging "INFO" "Does it continue? [Y|y]:" "0"
				read -t 30 enter
				if [ "$(grep -Eic "^Y$|^y$" <<< "${enter}")x" == "0x" ]
				then
					f_logging "ERROR" "The instance port[${mysql_recover_port}] to be restored is the same as the original instance port[${old_mysql_port}], and your choice is not [Y|y]" "2" "0"
					return
				fi
			fi
			#moshan:判断是否存在netstat, lsof工具, 并进行相应的端口检查判断, 最后发现端口冲突的话则退出本次恢复
			if [ "$(which netstat 2>/dev/null|wc -l)x" == "1x" ]
			then
				port_is_use="$(netstat -anplt |awk '{print $4}'|grep -c ":${mysql_recover_port}$")"
			elif [ "$(which lsof 2>/dev/null|wc -l)x" == "1x" ]
			then
				f_logging "WARN" "netstat: command not found. Please install in time..."|tee -a ${log_file}
				port_is_use="$(lsof -i:${mysql_recover_port}|wc -l)"
			else
				port_is_use=0
				f_logging "WARN" "lsof: command not found. Please install in time..."|tee -a ${log_file}
			fi
			if [ "${port_is_use}x" != "0x" ]
			then
				f_logging "WARN" "The port [${mysql_recover_port}] has been used and EXIT!" "2" "0"|tee -a ${log_file}
				return
			fi
			if [ ! -f "${mysql_base_dir}/bin/mysqld" ]
			then
				#moshan:当base目录中没有mysql文件, 则视为base目录为非完整的目录
				f_logging "ERROR" "MySQL base dir is error and EXIT!" "2" "0"|tee -a ${log_file}
				return
			fi
			
			mysql_error_file="$(grep -ic "^log[-_]error" ${mysql_conf} 2>/dev/null)"
			if [ "${mysql_error_file}x" == "0x" ]
			then
				#moshan:判断配置文件是否存在mysql error日志的配置, 若不存在则定义使用主机名并放在数据目录下
				mysql_error_file="${mysql_data_dir}/$(hostname).err"
			else
				#moshan:如果存在配置, 则取出路径
				mysql_error_file="$(grep -i "^log[-_]error" ${mysql_conf} 2>/dev/null|tail -1|tr -d " \t"|awk -F= '{print $2}')"
			fi
			if [ "$(grep -c "/" <<< "${mysql_error_file}")x" == "0x" ]
			then
				#moshan:格式化mysql error日志的路径, 如果不是绝对路径, 则添加绝对路径
				mysql_error_file="${mysql_data_dir}/${mysql_error_file}"
			fi
			#moshan:定义错误日志的目录路径
			mysql_error_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_error_file}")"
			[ ! -d "${mysql_error_dir}" ] && mkdir -p ${mysql_error_dir}
			
			if [ "$(grep -c "/" <<< "${mysql_pid}")x" == "0x" -o "${mysql_pid}x" == "x" ]
			then
				mysql_pid="${mysql_data_dir}/mysqld.pid"
			fi
			mysql_pid_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_pid}")"
			[ ! -d "${mysql_pid_dir}" ] && mkdir -p ${mysql_pid_dir}
			
			if [ "$(grep -c "/" <<< "${mysql_sock}")x" == "0x" -o "${mysql_sock}x" == "x" ]
			then
				mysql_sock="${mysql_data_dir}/mysqld.sock"
			fi
			#moshan:定义sock文件的目录路径
			mysql_sock_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_sock}")"
			[ ! -d "${mysql_sock_dir}" ] && mkdir -p ${mysql_sock_dir}
			
			mkdir -p ${mysql_data_dir} ${mysql_binlog_dir} ${mysql_tmp_dir} ${mysql_undo_dir}

			#moshan:定义mysql的安装目录
			mysql_install_dir="$(awk -F"/" '{print $NF}' <<< "${mysql_base_dir}")"
			if [ "${mysql_install_dir}x" != "x" ]
			then
				mysql_install_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_base_dir}")"
			else
				mysql_install_dir="$(awk -F'/' 'OFS="/",NF-=2' <<< "${mysql_base_dir}")"
			fi
			
			#moshan:根据原来的配置文件路径, 改成恢复实例的路径
			mysql_conf_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_conf_path}")"
			#moshan:如果是这种/my.cnf, 则mysql_conf_dir变量为空
			[ "${mysql_conf_dir}x" == "x" ] && mysql_conf_dir="/etc" 
			if [ "$(grep -c ${old_mysql_port} <<< "${mysql_conf_path}")" == "0x" ]
			then
				mysql_conf="${mysql_conf_dir}/my_${mysql_recover_port}.cnf"
			else
				mysql_conf="$(sed "s#${old_mysql_port}#${mysql_recover_port}#g" <<< ${mysql_conf_path})"
				mysql_conf_dir="$(awk -F'/' 'OFS="/",NF-=1' <<< "${mysql_conf}")"
				#moshan:如果是这种/my.cnf, 则mysql_conf_dir变量为空, 此时将路径变为/etc下
				[ "${mysql_conf_dir}x" == "x" ] && mysql_conf="/etc/${mysql_conf}"
			fi
			[ ! -d "${mysql_conf_dir}" ] && mkdir -p ${mysql_conf_dir}
			mv ${mysql_conf_tmp} ${mysql_conf} >/dev/null 2>&1
			if [ $? -ne 0 ]
			then
				f_logging "ERROR" "Execution failed for \"mv ${mysql_conf_tmp} ${mysql_conf}\"" "2" "0"|tee -a ${log_file}
				return
			fi
		else
			f_logging "ERROR" "mysql_port var is empty!" "2" "0"|tee -a ${log_file}
			return
		fi
	fi

	#moshan:判断mysql_data_dir_filesystem变量是否为空, 以及配置的分区是否正确
	if [ "${mysql_data_dir_filesystem}x" == "x" ] 
	then
		f_logging "ERROR" "mysql_data_dir_filesystem variable is empty!" "2" "0"|tee -a ${log_file}
		return
	fi

	#moshan:判断用户是否正确配置了mysql_data_dir_filesystem变量
	#moshan:检查磁盘空间, 检查数据目录所在的分区的剩余空间是否大于当前需要恢复的mysql的数据目录的大小
	mysql_recover_space="$(du -sm ${current_full_backup_dir} 2> /dev/null|awk -v size="${recover_disk_space}" '{print $1*size/1024}')"
	free_space="$(df|grep ${mysql_data_dir_filesystem} 2> /dev/null|awk '{print $4/1024/1024}')"
	if [ "${free_space}x" == "x" ]
	then
		f_logging "ERROR" "mysql_data_dir_filesystem=\"${mysql_data_dir_filesystem}\"." "2" "0"|tee -a ${log_file} 
		f_logging "ERROR" "This file system does not exist!" "2" "0"|tee -a ${log_file} 
		f_logging "ERROR" "Please update the configuration \"/etc/gfdmp.conf\" file and restart." "2" "0"|tee -a ${log_file}
		return
	else
		f_logging "INFO" "FileSystem \"[${mysql_data_dir_filesystem}]\" disk space:${free_space}G"|tee -a ${log_file}	
		f_logging "INFO" "The backup file size:$(du -sm ${current_full_backup_dir} 2> /dev/null|awk '{print $1/1024}')G"|tee -a ${log_file}
		f_logging "INFO" "This recovery may require five times the size of the backup file:${mysql_recover_space}G"|tee -a ${log_file}	
		free_space="$(awk -F. '{print $1}' <<< ${free_space})"
		mysql_recover_space="$(awk -F. '{print $1}' <<< ${mysql_recover_space})"
		if [ "$(grep -c "-" <<< "$((${free_space}-${mysql_recover_space}))")x" == "1x" ]
		then
			f_logging "ERROR" "FileSystem \"[${mysql_data_dir_filesystem}]\" not enough disk space." "2" "0"|tee -a ${log_file}
			return
		fi
	fi
	f_logging "INFO" "FULL >> ${mysql_recover_file}:xbstream start extracting data..."|tee -a ${log_file}
	command_tmp="xbstream -x < ${mysql_recover_file} -C ${backup_tmp_dir1}"
	if [ -f "${HOME}/mysql_recover_${mysql_port}" ]
	then
		f_logging "WARN" "There are already other processes recovering the instance[MySQL:${mysql_port}], please do not repeat the recovery." "0"|tee -a ${log_file}
		f_logging "WARN" "Manually delete files [${HOME}/mysql_recover_${mysql_port}] if there is no recovery process."|tee -a ${log_file}
		exit
	else
		touch ${HOME}/mysql_recover_${mysql_port}
	fi
	if [ "${debug}x" == "1x" ]
	then
		f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
	fi
	eval ${command_tmp}
	if [ $? -ne 0 ]
	then
		rm -f ${HOME}/mysql_recover_${mysql_port}
		f_logging "ERROR" "FULL >> ${mysql_recover_file}:xbstream failed to extract data!" "2" "0"|tee -a ${log_file}
		return
	else 	
		f_logging "INFO" "FULL >> ${mysql_recover_file}:xbstream extract data completion!"|tee -a ${log_file}
	fi
	if [ "${backup_encrypt}x" == "1x" ]
	then
		#moshan:判断配置文件如果是配了加密了, 则进行该部分的逻辑处理
		f_logging "INFO" "FULL >> ${mysql_recover_file}:xbcrypt start decrypting data..."|tee -a ${log_file}
		if [ "${encrypt_passwd}x" != "x" ]
		then
			encrypt_passwd=$(md5sum <<< "${encrypt_passwd}"|awk '{print $1}')
		else				
			encrypt_passwd=$(md5sum <<< "${mysql_recover_file}"|awk '{print $1}')
		fi
		#moshan:完成解密相关参数的拼接
		encrypt_tmp="--encrypt-algo=AES256 --encrypt-key=${encrypt_passwd}"
		cd ${backup_tmp_dir1}
		if [ "${debug}x" == "1x" ]
		then
			f_logging "RECOVER" "xbcrypt -d $(echo ${encrypt_tmp}|sed "s/--encrypt-key=.*/--encrypt-key='******'/g") < xbcrypt_file.xbcrypt > qp_file.qp && rm -f xbcrypt_file.xbcrypt"|tee -a ${log_file}
		fi
		for ii in $(find . -iname "*\.xbcrypt")
		do
			#moshan:解密命令的拼接
			command_tmp="xbcrypt -d ${encrypt_tmp} < ${ii} > $(dirname ${ii})/$(basename ${ii} .xbcrypt) && { f_logging \"INFO\" \"FULL >> ${ii}:Decryption completed\" && rm -f ${ii};} || { echo 0 >> ${backup_tmp_dir2}/tmp.log && f_logging \"ERROR\" \"FULL >> ${ii}:Unable to decrypt the file, the password may be wrong, please check.\" \"2\" \"0\"|tee -a;}"
			f_logging "INFO" "decrypting ${ii}"
			eval ${command_tmp} &
			[ "$([ -f "${backup_tmp_dir2}/tmp.log" ] && wc -l < ${backup_tmp_dir2}/tmp.log || echo 0)x" != "0x" ] && break
			while :
			do
				#moshan:多线程的解密逻辑, 线程书以threads参数的值为准
				#moshan:多线程逻辑的原理是吧解密命令放入后台, 并通过while去实时监控后台的解密进程数, 如果大于定义的threads的值则sleep, 反之进行解密
				xbcrypt_count=$(ps -ef|grep -v grep|grep -c -- "xbcrypt -d --encrypt-algo=AES256 --encrypt-key=")
					[ ${xbcrypt_count} -ge ${threads} ] && sleep 1 || break
			done
		done
	fi
	sleep 1
	while :
	do
		#moshan:这个while循环判断最后一个解密进程是否执行完成
		xbcrypt_count=$(ps -ef|grep -v grep|grep -c -- "xbcrypt -d --encrypt-algo=AES256 --encrypt-key=")
		[ "${xbcrypt_count}x" != "0x" ] && sleep 1 && continue
		sleep 2
		break
	done
	if [ "$([ -f "${backup_tmp_dir2}/tmp.log" ] && wc -l < ${backup_tmp_dir2}/tmp.log || echo 0)x" == "0x" ]
	then
		f_logging "INFO" "FULL >> ${mysql_recover_file}:xbcrypt decrypted data completion!"|tee -a ${log_file}
		f_logging "INFO" "FULL >> ${mysql_recover_file}:qpress start decompressing data..."|tee -a ${log_file}
	else
		[ -f "${backup_tmp_dir2}/tmp.log" ] && rm -f ${backup_tmp_dir2}/tmp.log
		f_logging "ERROR" "FULL >> Decryption failed!" "2" "0"|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		return
	fi
	decompress_state=0
	command_tmp="${backup_tool} --decompress --parallel=${threads} --use-memory=${use_mem} --tmpdir=${backup_tmp_dir2} ${backup_tmp_dir1}"
	if [ "${debug}x" == "1x" ]
	then
		f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
	fi
	if [ "${go_all_out}x" != "1x" ]
	then
		#moshan:go_all_out=1参数表示使用系统的所有资源进行qp文件解压
		#moshan:如果非1则使用xtrabackup的多线程解压
		eval ${command_tmp}
		if [ $? -eq 0 ]
		then
			#moshan:删除qp文件
			decompress_state=1
			if [ "${remove_qp_file}x" == "1x" ]
			then
				cd ${backup_tmp_dir1} && find ./ -iname "*\.qp"|xargs -i rm -f {}
			fi
		fi
	else
		#moshan:如果go_all_out=1则使用系统的qpress工具进行多线程解压, 使用系统的所有资源进行解压
		cd ${backup_tmp_dir1}
		for qpress_file in $(find ./ -iname "*\.qp")
		do
			f_logging "INFO" "qpress start decompressing file:${backup_tmp_dir1}/$(sed "s#./##g" <<< "${qpress_file}")"
			{ qpress -dT2 ${qpress_file}  $(dirname ${qpress_file}) && rm -f ${qpress_file};}&
		done
		wait
		[ $? -eq 0 ] && decompress_state=1
	fi
	if [ "${decompress_state}x" == "0x" ]
	then
		f_logging "ERROR" "FULL >> ${mysql_recover_file}:qpress decompressed data failed!" "2" "0"|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		return
	else
		f_logging "INFO" "FULL >> ${mysql_recover_file}:qpress decompressed data completion!"|tee -a ${log_file}
	fi
	f_logging "INFO" "FULL >> ${mysql_recover_file}:Start applying redo log..."|tee -a ${log_file}
	if [ "${#incremental_arry[@]}x" == "0x" ]
	then
		command_tmp="${backup_tool} --apply-log ${redo_only} --use-memory=${use_mem} --tmpdir=${backup_tmp_dir3} ${backup_tmp_dir1}"
		if [ "${debug}x" == "1x" ]
		then
			f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
		fi
		eval ${command_tmp}
		if [ $? -ne 0 ]
		then
			f_logging "ERROR" "FULL >> ${mysql_recover_file}:apply redo log error!" "2" "0"|tee -a ${log_file}
			rm -f ${HOME}/mysql_recover_${mysql_port}
			return
		else
			f_logging "INFO" "FULL >> ${mysql_recover_file}:apply redo log successed!"|tee -a ${log_file}
		fi
	else
		f_logging "INFO" "FULL >> Check to have incremental backup data, and only apply redo..."|tee -a ${log_file}
		command_tmp="${backup_tool} --defaults-file=${mysql_conf} --apply-log --redo-only --use-memory=${use_mem} --tmpdir=${backup_tmp_dir3} ${backup_tmp_dir1}"
		if [ "${debug}x" == "1x" ]
		then
			f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
		fi
		eval ${command_tmp}
		if [ $? -ne 0 ]
		then
			f_logging "ERROR" "FULL >> ${mysql_recover_file}:apply redo log error!" "2" "0"|tee -a ${log_file}
			rm -f ${HOME}/mysql_recover_${mysql_port}
			return
		else
			f_logging "INFO" "FULL >> ${mysql_recover_file}:apply redo log successed!"|tee -a ${log_file}
		fi
		for ((i=0;i<${#incremental_arry[@]};i++))
		do
			#moshan:如果存在增量备份文件, 则判断是否是最后一份增量, 如果不是最后一份, 则在应用redo的时候需要加上redo-only参数
			[ "$((${i}+1))x" == "${#incremental_arry[@]}x" ] && redo_only="" || redo_only="--redo-only"
			f_logging "INFO" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:xbstream start extracting data..."|tee -a ${log_file}
			command_tmp="xbstream -x < ${current_full_backup_dir}/${incremental_arry[${i}]} -C ${backup_tmp_dir2}"
			if [ "${debug}x" == "1x" ]
			then
				f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
			fi
			eval ${command_tmp}
			if [ $? -ne 0 ]
			then
				f_logging "ERROR" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:xbstream failed to extract data!" "2" "0"|tee -a ${log_file}
				rm -f ${HOME}/mysql_recover_${mysql_port}
				return
			else
				f_logging "INFO" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:xbstream extract data completion!"|tee -a ${log_file}
				if [ "${debug}x" == "1x" ]
				then
					f_logging "RECOVER" "xbcrypt -d ${encrypt_tmp} < xbcrypt_file.xbcrypt > qp_file.qp && rm -f xbcrypt_file.xbcrypt"|tee -a ${log_file}
				fi
				encrypt_tmp="--encrypt-algo=AES256 --encrypt-key=${encrypt_passwd}"
				cd ${backup_tmp_dir2}
				for ii in $(find . -iname "*\.xbcrypt")
				do
					#moshan:解密命令的拼接
					#command_tmp="xbcrypt -d ${encrypt_tmp} < ${ii} > $(dirname ${ii})/$(basename ${ii} .xbcrypt) && rm -f ${ii}"
					#command_tmp="xbcrypt -d ${encrypt_tmp} < ${ii} > $(dirname ${ii})/$(basename ${ii} .xbcrypt) && rm -f ${ii} || f_logging \"ERROR\" \"INCREMEMT >> ${ii}:Unable to decrypt the file, the password may be wrong, please check.\" \"2\" \"0\"|tee -a"
					command_tmp="xbcrypt -d ${encrypt_tmp} < ${ii} > $(dirname ${ii})/$(basename ${ii} .xbcrypt) && { f_logging \"INFO\" \"INCREMEMT >> ${ii}:Decryption completed\" && rm -f ${ii};} || { echo 0 >> ${backup_tmp_dir2}/tmp.log && f_logging \"ERROR\" \"INCREMEMT >> ${ii}:Unable to decrypt the file, the password may be wrong, please check.\" \"2\" \"0\"|tee -a;}"
					f_logging "INFO" "decrypting ${ii}"
					eval ${command_tmp} &
					[ "$([ -f "${backup_tmp_dir2}/tmp.log" ] && wc -l < ${backup_tmp_dir2}/tmp.log || echo 0)x" != "0x" ] && break
					while :
					do
						#moshan:多线程的解密逻辑, 线程数以threads参数的值为准
						#moshan:多线程逻辑的原理是把解密命令放入后台, 并通过while去实时监控后台的解密进程数, 如果大于定义的threads的值则sleep, 反之进行解密
						xbcrypt_count=$(ps -ef|grep -v grep|grep -c -- "xbcrypt -d --encrypt-algo=AES256 --encrypt-key=")
						[ ${xbcrypt_count} -ge ${threads} ] && sleep 1 || break
					done
				done
			fi
			while :
			do
				#moshan:这个while循环判断最后一个解密进程是否执行完成
				xbcrypt_count=$(ps -ef|grep -v grep|grep -c -- "xbcrypt -d --encrypt-algo=AES256 --encrypt-key=")
				[ "${xbcrypt_count}x" != "0x" ] && sleep 1 && continue
				sleep 2
				break
			done
			if [ "$([ -f "${backup_tmp_dir2}/tmp.log" ] && wc -l < ${backup_tmp_dir2}/tmp.log || echo 0)x" == "0x" ]
			then
				f_logging "INFO" "INCREMEMT >> ${mysql_recover_file}:xbcrypt decrypted data completion!"|tee -a ${log_file}
				f_logging "INFO" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:qpress start decompressing data..."|tee -a ${log_file}
			else
				[ -f "${backup_tmp_dir2}/tmp.log" ] && rm -f ${backup_tmp_dir2}/tmp.log
				f_logging "ERROR" "INCREMEMT >> Decryption failed!" "2" "0"|tee -a ${log_file}
				rm -f ${HOME}/mysql_recover_${mysql_port}
				return
			fi
			decompress_state=1
			command_tmp="${backup_tool} --decompress --parallel=${threads} --use-memory=${use_mem} --tmpdir=${backup_tmp_dir3} ${backup_tmp_dir2}"
			if [ "${debug}x" == "1x" ]
			then
				f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
			fi
			if [ "${go_all_out}x" != "1x" ]
			then
				#moshan:go_all_out=1参数表示使用系统的所有资源进行qp文件解压
				#moshan:如果非1则使用xtrabackup的多线程解压
				eval ${command_tmp}
				if [ $? -eq 0 ]
				then
					#moshan:删除qp文件
					decompress_state=1
					if [ "${remove_qp_file}x" == "1x" ]
					then
						cd ${backup_tmp_dir1} && find ./ -iname "*\.qp"|xargs -i rm -f {}
					fi
				fi
			else
				#moshan:如果go_all_out=1则使用系统的qpress工具进行多线程解压, 使用系统的所有资源进行解压
				cd ${backup_tmp_dir1}
				for qpress_file in $(find ./ -iname "*\.qp")
				do
					f_logging "INFO" "qpress start decompressing file:${backup_tmp_dir1}/$(sed "s#./##g" <<< "${qpress_file}")"
					{ qpress -dT2 ${qpress_file}  $(dirname ${qpress_file}) && rm -f ${qpress_file};}&
				done
				wait
				[ $? -eq 0 ] && decompress_state=1
			fi
			if [ "${decompress_state}x" == "0x" ]
			then
				f_logging "ERROR" "INCREMEMT >> ${mysql_recover_file}:qpress decompressed data failed!" "2" "0"|tee -a ${log_file}
				rm -f ${HOME}/mysql_recover_${mysql_port}
				return
			else
				f_logging "INFO" "INCREMEMT >> ${mysql_recover_file}:qpress decompressed data completion!"|tee -a ${log_file}
			fi
			f_logging "INFO" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:Start applying redo log..."|tee -a ${log_file}
			command_tmp="${backup_tool} --defaults-file=${mysql_conf} --apply-log ${redo_only} --incremental-dir=${backup_tmp_dir2} --use-memory=${use_mem} --tmpdir=${backup_tmp_dir3} ${backup_tmp_dir1}"
			if [ "${debug}x" == "1x" ]
			then
				f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
			fi
			eval ${command_tmp}
			if [ $? -ne 0 ]
			then
				f_logging "ERROR" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:apply redo log error!" "2" "0"|tee -a ${log_file}
				rm -f ${HOME}/mysql_recover_${mysql_port}
				return
			else
				f_logging "INFO" "INCREMEMT >> ${current_full_backup_dir}/${incremental_arry[${i}]}:apply redo log successed!"|tee -a ${log_file}
			fi
			[ -d "${backup_tmp_dir2}" ] && rm -rf ${backup_tmp_dir2}/*
		done
	fi

	#moshan:判断用户是否正确配置了mysql_data_dir_filesystem变量
	#moshan:检查磁盘空间, 检查数据目录所在的分区的剩余空间是否大于当前需要恢复的mysql的数据目录的大小
	mysql_data_dir_space="$(du -sm ${backup_tmp_dir1} 2> /dev/null|awk '{print $1/1024}')"
	free_space="$(df|grep ${mysql_data_dir_filesystem} 2> /dev/null|awk '{print $4/1024/1024}')"
	if [ "${free_space}x" == "x" ]
	then
		f_logging "ERROR" "FileSystem \"[${mysql_data_dir_filesystem}]\" does not exist" "2" "0"|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		return
	else
		f_logging "INFO" "FileSystem \"[${mysql_data_dir_filesystem}]\" disk space:${free_space}G"|tee -a ${log_file}	
		f_logging "INFO" "MySQL data dir space:${mysql_data_dir_space}G"|tee -a ${log_file}	
		free_space="$(awk -F. '{print $1}' <<< ${free_space})"
		mysql_data_dir_space="$(awk -F. '{print $1}' <<< ${mysql_data_dir_space})"
		if [ "$(grep -c "-" <<< "$((${free_space}-${mysql_data_dir_space}))")x" == "1x" ]
		then
			#moshan:判断磁盘空间
			f_test_dir_partition "${backup_tmp_dir1}"
			if [ "${test_dir_par}x" != "${mysql_data_dir_filesystem}x" ]
			then
				#moshan:如果磁盘空间不足, 且恢复的目录与数据目录不在一个分区, 则不继续进行恢复操作
				f_logging "ERROR" "FileSystem \"[${mysql_data_dir_filesystem}]\" not enough disk space" "2" "0"|tee -a ${log_file}
				rm -f ${HOME}/mysql_recover_${mysql_port}
				return
			fi
		fi
	fi
	if [ -d "${mysql_binlog_dir}" ]
	then
		rm -rf ${mysql_binlog_dir}/* 2>/dev/null
	else
		mkdir -p ${mysql_binlog_dir}
	fi
	if [ "${copy_file_mode}x" == "copyx" ]
	then
		copy_opt="--copy-back"
	else
		copy_opt="--move-back"
	fi
	command_tmp="${backup_tool} --defaults-file=${mysql_conf} ${copy_opt} --use-memory=${use_mem} --tmpdir=${backup_tmp_dir4} ${backup_tmp_dir1}"

	if [ -d "${mysql_data_dir}" ]
	then
		if [ "$(ls ${mysql_data_dir}/* 2>/dev/null|wc -l)x" != "0x" ]
		then
			if [ "${force_copy_file}x" == "1x" ]
			then
				f_logging "WARN" "The data directory[\"${mysql_data_dir}\"] is not empty, but the \"--force-copy=1 or -fc=1\" parameter has been used, so now the data directory is forced to be replaced."
				rm -rf ${mysql_data_dir}/* 2>/dev/null
			else
				f_logging "WARN" "The data directory[\"${mysql_data_dir}\"] is not empty, but the \"--force-copy=1 or -fc=1\" parameter is not used, so the data directory will not be replaced now."
				f_logging "INFO" "Please manually clear the data directory and copy it using the command.[${command_tmp}]"
				f_logging "INFO" "After waiting for the data directory to be copied successfully, please manually change the permissions of the data directory and start MySQL."
				rm -f ${HOME}/mysql_recover_${mysql_port}
				exit
			fi
		fi
	else
		mkdir -p ${mysql_data_dir}
	fi

	if [ "${debug}x" == "1x" ]
	then
		f_logging "RECOVER" "${command_tmp}"|tee -a ${log_file}
	fi
	eval ${command_tmp}
	if [ $? -ne 0 ]
	then
		f_logging "ERROR" "FULL and INCREMEMT >> copy back error!" "2" "1"|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		return
	else
		f_logging "INFO" "FULL and INCREMEMT >> copy back successed!"|tee -a ${log_file}
	fi
	f_logging "INFO" "MySQL data recovery success,and starting..."|tee -a ${log_file}
	[ ! -f "${mysql_error_file}" ] && touch ${mysql_error_file}
	[ "$(id ${start_mysql_user} 2>/dev/null|wc -l)x" == "0x" ] && useradd ${start_mysql_user} 2>/dev/null
	if [ "${mysql_tmp_dir}x" == "/tmpx" ]
	then
		#moshan:如果mysql的tmp目录是"/tmp, 则不对该目录进行权限修改"
		chown -R ${start_mysql_user}. ${mysql_error_dir} ${mysql_pid_dir} ${mysql_sock_dir} ${mysql_data_dir} ${mysql_binlog_dir} ${mysql_undo_dir} 2>/dev/null
	else
		chown -R ${start_mysql_user}. ${mysql_error_dir} ${mysql_pid_dir} ${mysql_sock_dir} ${mysql_data_dir} ${mysql_binlog_dir} ${mysql_tmp_dir} ${mysql_undo_dir}  2>/dev/null
	fi
	echo "cd ${mysql_base_dir} && ./bin/mysqld_safe --defaults-file=${mysql_conf} --pid-file=${mysql_pid} --socket=${mysql_sock} --log-error=${mysql_error_file} --user=${start_mysql_user} >/dev/null 2>&1 &" > ${mysql_data_dir}/start.sh
	f_logging "INFO" "Start the mysql command: \"bash ${mysql_data_dir}/start.sh\"" |tee -a ${log_file}
	bash ${mysql_data_dir}/start.sh >/dev/null 2>&1
	sleep 2
	tmp=0
	rm -f ${HOME}/mysql_recover_${mysql_port}
	while :
	do
		#moshan:检查mysql启动状态
		if [ ${tmp} -gt 30 ]
		then
			f_logging "ERROR" "MySQL failed to start..." "2" "0"|tee -a ${log_file}
			f_logging "ERROR" "The error information is as follows..." "2" "0"|tee -a ${log_file}
			[ -f "${mysql_error_file}" ] && grep -i error ${mysql_error_file}
			f_logging "ERROR" "For more error information, please read \"${mysql_error_file}\" file" "2" "0"|tee -a ${log_file}
			rm -f ${HOME}/mysql_recover_${mysql_port}
			return
		fi
		${mysql_base_dir}/bin/mysqladmin -u${mysql_admin_user} -p${mysql_admin_passwd} -S ${mysql_sock} ping >/dev/null 2>&1
		if [ $? -eq 0 ]
		then
			f_logging "INFO" "MySQL is alive..."|tee -a ${log_file}
			break
		else
			sleep 2
			let tmp+=1
		fi
	done
	f_logging "INFO" "MySQL data recovery successed..."|tee -a ${log_file}

	#moshan:下面是建立复制, 该部分代码属于重复, 已经将该代码写成函数, 待日后重构的时候再优化
	if [ "${repl}x" == "x" ]
	then
		echo -e "\033[0m"
		if [ "${copy_file_mode}x" == "copyx" ]
		then
			#moshan:不删除数据文件
			f_logging "INFO" "Please manually delete the temporary data directory[${backup_tmp_dir1}]."|tee -a ${log_file}
			return "0"
		else
			return "1"
		fi
	fi
	f_logging "INFO" "Creating replication relationship from MySQL master..."|tee -a ${log_file}
	slave_info_file="${mysql_data_dir}/xtrabackup_info"
	if [ "${repl_info[5]}x" == "masterx" ]
	then
		slave_info_file="${mysql_data_dir}/xtrabackup_slave_info"
		if [ ! -f "${slave_info_file}" ]
		then
			f_logging "WARN" "repl_info[5]=${repl_info[5]}, but [${slave_info_file}]: No such file ..."|tee -a ${log_file}
			f_logging "ERROR" "Create replication relationship error..." "1" "0"|tee -a ${log_file}
			rm -f ${HOME}/mysql_recover_${mysql_port}
			return
		fi
	elif [ ! -f "${slave_info_file}" ]
	then
		f_logging "WARN" "repl_info[5]=${repl_info[5]}, but [${slave_info_file}]: No such file ..."|tee -a ${log_file}
		f_logging "ERROR" "Create replication relationship error..." "1" "0"|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		return
	fi
	#moshan:判断是否开启了gtid
	mysql_sock="$(tr -d " \t" < ${mysql_conf}|grep -i "^socket="|tail -1|awk -F= '{print $2}')"
	mysql_comm="${mysql_base_dir}/bin/mysql -u${mysql_admin_user} -p${mysql_admin_passwd} -S ${mysql_sock}"       #定义mysql的连接命令，通过sock连接
	gtid_mode_state="$(${mysql_comm} -NBe "show variables like 'gtid_mode';" 2>/dev/null|grep -ic on)"
	enforce_gtid_consistency_state="$(${mysql_comm} -NBe "show variables like 'enforce_gtid_consistency';" 2>/dev/null|grep -ic on)"
	if [ "${gtid_mode_state}x" == "1x" -a "${enforce_gtid_consistency_state}x" == "1x" ]
	then
		#moshan:如果开启了gtid, 则使用gtid去复制
		if [ "${repl_info[5]}x" == "masterx" ]
		then
			comm="stop slave;reset slave all;reset master;$(grep "SET GLOBAL gtid_purged=" ${slave_info_file})"
		else
			sed -n '/change/,/innodb_from_lsn/p' ${slave_info_file}|sed '$d'|tr -s "\n" "\ "|awk -F'change' '{print $2}'|tr -d "\'" >${master_gtid_info_file}
			comm="stop slave;reset slave all;reset master;set global gtid_purged='$(cat ${master_gtid_info_file})';"
		fi
		${mysql_comm} -NBe "${comm}" 2> /dev/null
		repl_comm="'change master to master_host=\"${repl_info[0]}\",master_user=\"${repl_info[1]}\",master_password=\"${repl_info[2]}\",master_port=${repl_info[3]},master_auto_position=1;start slave;'"
	else
		#moshan:如果没有开启gtid, 则使用position, 去复制
		${mysql_comm} -NBe "SET @@GLOBAL.GTID_MODE = ON_PERMISSIVE;SET @@GLOBAL.GTID_MODE = OFF_PERMISSIVE;SET @@GLOBAL.GTID_MODE = off;SET @@GLOBAL.ENFORCE_GTID_CONSISTENCY = off;" > /dev/null 2>&1
		if [ "${repl_info[5]}x" == "masterx" ]
		then
			repl_comm="'stop slave;reset slave all;change master to master_host=\"${repl_info[0]}\",master_user=\"${repl_info[1]}\",master_password=\"${repl_info[2]}\",master_port=${repl_info[3]},$(awk -F"CHANGE MASTER TO" '{print $2}' ${slave_info_file});start slave;'"
		else
			mysql_binlog_file_name="$(grep -i "binlog_pos = filename" ${slave_info_file}|awk -F"'" '{print $2}')"
			mysql_binlog_pos="$(grep -i "binlog_pos = filename" ${slave_info_file}|awk -F"'" '{print $4}')"
			repl_comm="'stop slave;reset slave all;change master to master_host=\"${repl_info[0]}\",master_user=\"${repl_info[1]}\",master_password=\"${repl_info[2]}\",master_port=${repl_info[3]},master_log_file=\"${mysql_binlog_file_name}\",master_log_pos=${mysql_binlog_pos};start slave;'"
		fi
	fi
	echo "${mysql_comm} -NBe ${repl_comm} >/dev/null 2>&1" > ${backup_tmp_dir2}/tmp.log
	bash ${backup_tmp_dir2}/tmp.log
	repl_info_tmp="$(awk -F"change master to" '{print $2}' ${backup_tmp_dir2}/tmp.log|awk -F"start slave" '{print "change master to"$1}')"
	sleep 2
	#moshan:检查复制状态
	${mysql_comm} -e "show slave status\G" 2>/dev/null|grep -Ei "Running:" > ${backup_tmp_dir2}/tmp.log
	if [ "$(grep -ci ": Yes" ${backup_tmp_dir2}/tmp.log)x" == "2x" ]
	then
		f_logging "INFO" "${repl_info_tmp}"|tee -a ${log_file}
		cat ${backup_tmp_dir2}/tmp.log|tee -a ${log_file}
		f_logging "INFO" "Replication relationship from MySQL master been created..."|tee -a ${log_file}
	else
		f_logging "ERROR" "Create replication relationship error..." "1" "0"|tee -a ${log_file}
		f_logging "ERROR" "${repl_info_tmp}" "2" "0"|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		${mysql_comm} -e "show slave status\G" 2>/dev/null|grep -Ei "last_io_err..:|last_sql_err..:"|tee -a ${log_file}
	fi
	[ -f "${master_gtid_info_file}" ] && rm -f ${master_gtid_info_file}
	[ -f "${backup_tmp_dir2}/tmp.log" ] && rm -f ${backup_tmp_dir2}/tmp.log
	[ -f "${backup_tmp_dir1}/tmp.log" ] && rm -f ${backup_tmp_dir1}/tmp.log
	if [ "${copy_file_mode}x" == "copyx" ]
	then
		#moshan:不删除数据文件
		f_logging "INFO" "Please manually delete the temporary data directory[${backup_tmp_dir1}]."|tee -a ${log_file}
		rm -f ${HOME}/mysql_recover_${mysql_port}
		return "0"
	else
		return "1"
	fi
}
